encoding.py 1.3 KB

1234567891011121314151617181920212223242526272829303132333435363738394041
  1. import codecs
  2. import locale
  3. import re
  4. import sys
  5. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  6. if MYPY_CHECK_RUNNING:
  7. from typing import List, Tuple, Text
  8. BOMS = [
  9. (codecs.BOM_UTF8, 'utf-8'),
  10. (codecs.BOM_UTF16, 'utf-16'),
  11. (codecs.BOM_UTF16_BE, 'utf-16-be'),
  12. (codecs.BOM_UTF16_LE, 'utf-16-le'),
  13. (codecs.BOM_UTF32, 'utf-32'),
  14. (codecs.BOM_UTF32_BE, 'utf-32-be'),
  15. (codecs.BOM_UTF32_LE, 'utf-32-le'),
  16. ] # type: List[Tuple[bytes, Text]]
  17. ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)')
  18. def auto_decode(data):
  19. # type: (bytes) -> Text
  20. """Check a bytes string for a BOM to correctly detect the encoding
  21. Fallback to locale.getpreferredencoding(False) like open() on Python3"""
  22. for bom, encoding in BOMS:
  23. if data.startswith(bom):
  24. return data[len(bom):].decode(encoding)
  25. # Lets check the first two lines as in PEP263
  26. for line in data.split(b'\n')[:2]:
  27. if line[0:1] == b'#' and ENCODING_RE.search(line):
  28. result = ENCODING_RE.search(line)
  29. assert result is not None
  30. encoding = result.groups()[0].decode('ascii')
  31. return data.decode(encoding)
  32. return data.decode(
  33. locale.getpreferredencoding(False) or sys.getdefaultencoding(),
  34. )